import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomText;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

import java.sql.Connection;

import com.mysql.jdbc.jdbc2.optional.MysqlConnectionPoolDataSource;
import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.NlpAnalysis;

import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

public class SingleThreadCatch {
    public static void main(String[] args) throws IOException, NoSuchAlgorithmException, SQLException {
        WebClient client = new WebClient(BrowserVersion.CHROME);
        client.getOptions().setJavaScriptEnabled(false);
        client.getOptions().setCssEnabled(false);

        String baseUrl = "https://so.gushiwen.cn";
        String pathUrl = "/gushi/tangshi.aspx";

        List<String> detailUrlList = new ArrayList<>();
        //列表页的解析
        {
            String url = baseUrl + pathUrl;
            HtmlPage page = client.getPage(url);
            List<HtmlElement> divs = page.getBody().getElementsByAttribute("div", "class", "typecont");
            for (HtmlElement div : divs) {
                List<HtmlElement> as = div.getElementsByTagName("a");
                for (HtmlElement a : as) {
                    String detailUrl = a.getAttribute("href");
                    detailUrlList.add(baseUrl + detailUrl);
                }
            }
        }

        MysqlConnectionPoolDataSource dataSource = new MysqlConnectionPoolDataSource();//连接池,好处参照线程池
        dataSource.setServerName("127.0.0.1");
        dataSource.setPort(3306);
        dataSource.setUser("root");
        dataSource.setPassword("missh111");
        dataSource.setDatabaseName("java20210125");
        dataSource.setUseSSL(false);
        dataSource.setCharacterEncoding("UTF8");
        MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");

        Connection connection = dataSource.getConnection();
        String sql = "Insert INTO tangshi" +
                "(sha256,dynasty,title,author,content,words)"
                + "VALUES(?,?,?,?,?,?)";
        PreparedStatement statement = connection.prepareStatement(sql);

        //详情页的请求和解析
        for (String url : detailUrlList) {
            HtmlPage page = client.getPage(url);
            String xpath;
            DomText domText;
            xpath = "//div[@class='cont']/h1/text()";
            domText = (DomText) page.getBody().getByXPath(xpath).get(0);
            String title = domText.asText();

            xpath = "//div[@class='cont']/p[@class='source']/a[1]/text()";
            domText = (DomText) page.getBody().getByXPath(xpath).get(0);
            String dynasty = domText.asText();

            xpath = "//div[@class='cont']/p[@class='source']/a[2]/text()";
            domText = (DomText) page.getBody().getByXPath(xpath).get(0);
            String author = domText.asText().substring(1, domText.getLength() - 1);

            xpath = "//div[@class='cont']/div[@class='contson']";
            HtmlElement element = (HtmlElement) page.getBody().getByXPath(xpath).get(0);
            String content = element.getTextContent().trim();

            //1.计算sha-256
            String s = title + content;
            messageDigest.update(s.getBytes("UTF-8"));
            byte[] result = messageDigest.digest();
            StringBuilder sha256 = new StringBuilder();
            for (byte b : result) {
                sha256.append(String.format("%02x", b));
            }
            //2.计算分词
            List<Term> termList = new ArrayList<>();
            termList.addAll(NlpAnalysis.parse(title).getTerms());
            termList.addAll(NlpAnalysis.parse(content).getTerms());
            List<String> words = new ArrayList<>();
            for (Term term : termList) {
                if (term.getNatureStr().equals("w")) {//词性为w,剔除(跳过此次循环)
                    continue;
                }
                if (term.getNatureStr().equals("null")) {//分词之后为null,也剔除掉
                    continue;
                }
                if (term.getRealName().length() < 2) {//长度小于2,也过滤掉
                    continue;
                }
                words.add(term.getRealName());
            }
            String insertWords = String.join(",", words);//把words中的词用逗号连接起来
            statement.setString(1, sha256.toString());
            statement.setString(2, dynasty);
            statement.setString(3, title);
            statement.setString(4, author);
            statement.setString(5, content);
            statement.setString(6, insertWords);

            com.mysql.jdbc.PreparedStatement mysqlStatement= (com.mysql.jdbc.PreparedStatement) statement;
            System.out.println(mysqlStatement.asSql());

            statement.executeUpdate();

            //System.out.println(title + "插入成功");
        }
    }
}
